﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.IO;
using System.Configuration;
using System.Data;
using System.Data.OleDb;
using System.Data.Common;
using System.Collections;
using ICSharpCode;
using LumenWorks.Framework.IO.Csv;
using System.Threading;


using Common;

namespace ForexHistory
{
    public class CurrencyHistoryManager
    {
        #region Vars



        Queue<FileBuffer> _buffer;
        Queue<FileInfo> _loadQueue;

        CurrencyPairHistory[] _history;
        public string ToLoad;

        DateTime _started;
        private object _completeLock = new object();

        private int _start;

        public int Start
        {
            get { return _start; }
            set { _start = value; }
        }

        private int _end;

        public int End
        {
            get { return _end; }
            set { _end = value; }
        }

        #region Currency Load Flags
        private bool _loadUSDCHF = false;

        public bool LoadUSDCHF
        {
            get { return _loadUSDCHF; }
            set { _loadUSDCHF = value; }
        }

        private bool _loadGBPUSD = false;

        public bool LoadGBPUSD
        {
            get { return _loadGBPUSD; }
            set { _loadGBPUSD = value; }
        }

        private bool _loadUSDJPY = false;

        public bool LoadUSDJPY
        {
            get { return _loadUSDJPY; }
            set { _loadUSDJPY = value; }
        }

        private bool _loadEURUSD = false;

        public bool LoadEURUSD
        {
            get { return _loadEURUSD; }
            set { _loadEURUSD = value; }
        }

        #endregion
        #endregion

        public CurrencyHistoryManager()
        {

        }


        public void AddFile(string path)
        {
            // CurrencyPair TickTime RateBid RateAsk

            InvokeStatusEvent("Starting Data Load...", CurrencyPair.UNSUPPORTED);
            _started = DateTime.Now;



            // UNSUPPORTED will not be loaded
            _history = new CurrencyPairHistory[Enum.GetValues(typeof(CurrencyPair)).Length - 1];

            int i = 0;
            int factor = (_end - _start) + 1;

            foreach (CurrencyPair _c in Enum.GetValues(typeof(CurrencyPair)))
            {
                if (_c == CurrencyPair.UNSUPPORTED)
                    continue;

                if (LoadAllowed(_c))
                {
                    CurrencyPairHistory _toAdd = new CurrencyPairHistory(_c, _start, _end);
                    _toAdd.OnError += _OnError;
                    _toAdd.OnUpdateStatus += _OnUpdateStatus;
                    _toAdd.InitializeArray();
                    _toAdd.OnLoadCompleted += _toAdd_OnLoadCompleted;
                    _history[i] = _toAdd;
                    InvokeStatusEvent("Building Object for: " + _c.ToString(), _c);
                }

                ++i;
            }




            _loadQueue = new Queue<FileInfo>();

            FileInfo _f = new FileInfo(path);

            _loadQueue.Enqueue(_f);

        }

        public void SearchForSourceFiles()
        {
            // CurrencyPair TickTime RateBid RateAsk

            InvokeStatusEvent("Starting Data Load...", CurrencyPair.UNSUPPORTED);
            _started = DateTime.Now;


            // Build Pair Objects

            // UNSUPPORTED will not be loaded
            _history = new CurrencyPairHistory[Enum.GetValues(typeof(CurrencyPair)).Length - 1];

            int i = 0;
            int factor = (_end - _start) + 1;
            foreach (CurrencyPair _c in Enum.GetValues(typeof(CurrencyPair)))
            {
                if (_c == CurrencyPair.UNSUPPORTED)
                    continue;

                if (LoadAllowed(_c))
                {
                    CurrencyPairHistory _toAdd = new CurrencyPairHistory(_c, _start, _end);
                    _toAdd.OnError += _OnError;
                    _toAdd.OnUpdateStatus += _OnUpdateStatus;
                    _toAdd.InitializeArray();
                    _toAdd.OnLoadCompleted += _toAdd_OnLoadCompleted;
                    _history[i] = _toAdd;
                    InvokeStatusEvent("Building Object for: " + _c.ToString(), _c);
                }

                ++i;
            }


            DirectoryInfo _dif = new DirectoryInfo(ToLoad);

            _loadQueue = new Queue<FileInfo>();

            ProcessDirectory(_dif);

        }

        protected void _toAdd_OnLoadCompleted(object sender, EventArgs e)
        {
            InvokeLoadCompleteEvent(sender);
        }

        public Tick GetTick(CurrencyPair Pair, DateTime Time)
        {
            // Time.Year
            // Time. MOnth
            //Time.Day
            // Time.Hour
            return _history[(int)Pair].GetTick(Time);
        }


        public TickHistoryList GetHour(CurrencyPair _pair, DateTime date)
        {
            Tick[] _r = _history[(int)_pair].GetHour(date);

            TickHistoryList _res = new TickHistoryList();

            for (int i = 0; i < _r.Length; ++i)
            {
                _res.Add(_r[i]);
            }

            return _res;
        }

        #region File Loading

        private System.ComponentModel.BackgroundWorker _producer;
        private System.ComponentModel.BackgroundWorker[] _consumers;
        private System.ComponentModel.BackgroundWorker[] _cleanup;

        private SemaphoreSlim _workers;// = new SemaphoreSlim(3, 3);

        private bool _loadComplete = false;
        private bool _processingComplete = false;

        private object _processLock = new object();

        public bool ProcessingComplete
        {
            get
            {
                lock (_processLock)
                {
                    return _processingComplete;
                }
            }
            set
            {
                lock (_processLock)
                {
                    _processingComplete = value;
                }
            }
        }



        private object _bufferLock = new object();

        public void StartLoad()
        {
            _buffer = new Queue<FileBuffer>();


            InvokeStatusEvent("Using P/C Load...", CurrencyPair.UNSUPPORTED);
            _producer = new System.ComponentModel.BackgroundWorker();
            _producer.DoWork += _producer_DoWork;
            _producer.RunWorkerAsync();

            Object _t = ConfigurationManager.AppSettings.Get("LoaderThreadLimit");
            int _limit = 4;

            if (_t != null)
                _limit = Convert.ToInt32(_t);

            _workers = new SemaphoreSlim(_limit, _limit);


            _consumers = new System.ComponentModel.BackgroundWorker[_limit];


            for (int i = 0; i < _consumers.Length; ++i)
            {
                _workers.Wait();

                _consumers[i] = new System.ComponentModel.BackgroundWorker();
                _consumers[i].DoWork += _consumer_DoWork;

                //if (i == 0)
                //_consumers[i].RunWorkerCompleted += _consumer_RunWorkerCompleted;

                _consumers[i].RunWorkerAsync();
            }

            _cleanup = new System.ComponentModel.BackgroundWorker[_limit];


            Object _threadLimit = ConfigurationManager.AppSettings.Get("LoaderThreadLimit");
            int _tlimit = 4;

            if (_threadLimit != null)
                _tlimit = Convert.ToInt32(_threadLimit);

            for (int b = 0; b < _limit; ++b)
            {
                InvokeStatusEvent("Starting Cleanup: Waiting for worker " + (b + 1) + " of " + _limit + " to finish.", CurrencyPair.UNSUPPORTED);
                _workers.Wait();
            }

            InvokeStatusEvent("Starting Cleanup...", CurrencyPair.UNSUPPORTED);
            //_workers.Release(_limit);

            for (int i = 0; i < _cleanup.Length; ++i)
            {
                _cleanup[i] = new System.ComponentModel.BackgroundWorker();
                _cleanup[i].DoWork += _cleanup_DoWork;
                _cleanup[i].RunWorkerAsync();
            }


            for (int b = 0; b < _limit; ++b)
            {
                InvokeStatusEvent("Complete: Waiting for worker " + (b + 1) + " of " + _limit + " to finish.", CurrencyPair.UNSUPPORTED);
                _workers.Wait();
            }


            _processingComplete = true;

            InvokeStatusEvent("All Workers Complete", CurrencyPair.UNSUPPORTED);


        }

        protected void _consumer_DoWork(object sender, System.ComponentModel.DoWorkEventArgs e)
        {
            FileBuffer _temp;
            int i = 0;

            try
            {
                //_workers.Wait();

                // Safe?
                while (!_loadComplete || _buffer.Count > 0)
                {
                    try
                    {
                        if (_buffer.Count == 0)
                            continue;

                        lock (_bufferLock)
                        {
                            try
                            {
                                _temp = _buffer.Dequeue();
                            }
                            catch (Exception ex)
                            {
                                continue;
                            }
                        }

                        if (_temp != null)
                        {
                            int _lineCount = 0;

                            CurrencyPair _id = Shared.GetCurrencyPairFromFileName(_temp.Name);

                            if (_id == CurrencyPair.UNSUPPORTED)
                            {
                                InvokeErrorEvent("CurrencyType during load was 'UNSUPPORTED'.", CurrencyPair.UNSUPPORTED);
                                return;
                            }

                            if (_history[(int)_id] != null)
                            {
                                _lineCount = _history[(int)_id].ProcessFileFromMemory(_temp.ms, _temp.FullPath);



                            }
                            else
                                InvokeErrorEvent("_history[_id] was null for " + _id.ToString(), _id);



                            InvokeStatusEvent("Loaded " + _lineCount + " lines from " + _temp.FullPath, _id);
                        }

                        if (_temp != null)
                        {
                            _temp.ms.Dispose();
                            _temp.ms = null;
                            _temp = null;
                        }

                        if ((i % 10) == 0)
                            GC.Collect();

                        ++i;

                    }
                    catch (Exception ex)
                    {
                        InvokeErrorEvent(ex.Message, CurrencyPair.UNSUPPORTED);
                    }

                }

            }
            finally
            {
                _workers.Release();
            }

        }

        protected void _producer_DoWork(object sender, System.ComponentModel.DoWorkEventArgs e)
        {
            // Open and fill file
            _loadComplete = false;
            //foreach (string _fileName in _loadQueue)
            //for (int i = 0; i < _loadQueue.Count; ++i)
            int i = 0;

            while (!_loadComplete)
            {
                try
                {
                    FileInfo _f = null;

                    try
                    {
                        _f = _loadQueue.Dequeue();
                    }
                    catch (InvalidOperationException ex)
                    {
                        // queue empty
                        InvokeStatusEvent("Producer load finished.", CurrencyPair.UNSUPPORTED);
                        break;
                    }

                    MemoryStream ms = LoadFileIntoMemory(_f);

                    if (ms != null)
                    {
                        FileBuffer _b = new FileBuffer();
                        _b.ms = ms;
                        _b.FullPath = _f.FullName;
                        _b.Name = _f.Name;

                        lock (_bufferLock)
                        {
                            _buffer.Enqueue(_b);
                        }

                        InvokeErrorEvent("Queued: " + _f.FullName + "  Queue Count: " + _buffer.Count.ToString(), CurrencyPair.UNSUPPORTED);
                        ++i;

                    }

                    if ((i % 10) == 0)
                    {
                        GC.Collect();
                    }

                    while (_buffer.Count > 20)
                    {
                        // spin
                    }
                }
                catch (Exception ex)
                {
                    InvokeErrorEvent(ex.Message, CurrencyPair.UNSUPPORTED);
                }
            }

            lock (_bufferLock)
            {
                _loadComplete = true;
            }

            GC.Collect();

        }

        private MemoryStream LoadFileIntoMemory(FileInfo _f)
        {

            if ((_f.FullName.ToLower().Contains(".csv") || _f.FullName.ToLower().Contains(".zip")) && !_f.FullName.ToLower().Contains("lock"))
            {
                try
                {

                    if (_f.FullName.ToLower().Contains(".zip"))
                    {
                        MemoryStream _ms = Zip.MemExtract(_f.FullName);
                        return _ms;
                    }
                }
                catch (FileNotFoundException ex)
                {
                    InvokeErrorEvent(ex.Message, CurrencyPair.UNSUPPORTED);
                }
                catch (Exception ex)
                {
                    InvokeErrorEvent(ex.Message, CurrencyPair.UNSUPPORTED);
                }
                finally
                {
                    GC.Collect();
                }
            }
            else
            {
                InvokeStatusEvent("Skipping non-csv file: " + _f.FullName, CurrencyPair.UNSUPPORTED);
            }

            return null;
        }

        private void ProcessDirectory(DirectoryInfo _d)
        {

            foreach (FileInfo _f in _d.GetFiles())
            {
                if (LoadAllowed(_f.Name))
                {
                    _loadQueue.Enqueue(_f);
                }
            }

            foreach (DirectoryInfo _nd in _d.GetDirectories())
            {
                string _parent = _nd.Parent.Name.Trim().ToLower();
                string _current = _nd.Name.Trim().ToLower();
                string _root = ToLoad.Trim().ToLower();

                int _year = -1, _month = -1;

                try
                {

                    if (!Int32.TryParse(_parent, out _year))
                    {
                        _year = -1;

                        if (!Int32.TryParse(_current, out _year))
                        {
                            _year = -1;
                        }
                    }

                    if (!Int32.TryParse(_current, out _month))
                    {
                        _month = -1;
                    }
                    else
                    {
                        if ((_month > 12) || (_month < 1))
                            _month = -1;
                    }


                    // /2003
                    if (_month == -1 && _year >= _start && _year <= _end)
                    {
                        InvokeStatusEvent("Adding " + _year.ToString(), CurrencyPair.UNSUPPORTED);
                        ProcessDirectory(_nd);
                    }
                    else if ((_year == _start && _year == _end)/* && (_month >= Start.Month && _month <= End.Month)*/)
                    {
                        InvokeStatusEvent("Adding " + _month.ToString() + "/" + _year.ToString(), CurrencyPair.UNSUPPORTED);
                        ProcessDirectory(_nd);
                    }
                    else if (_year == _start /*&& _month >= Start.Month*/ && (_start != _end))
                    {
                        InvokeStatusEvent("Adding " + _month.ToString() + "/" + _year.ToString(), CurrencyPair.UNSUPPORTED);
                        ProcessDirectory(_nd);
                    }
                    else if (_year == _end /*&& _month <= End.Month*/ && (_start != _end))
                    {
                        InvokeStatusEvent("Adding " + _month.ToString() + "/" + _year.ToString(), CurrencyPair.UNSUPPORTED);
                        ProcessDirectory(_nd);
                    }
                    else if (_year > _start && _year < _end)
                    {
                        InvokeStatusEvent("Adding " + _month.ToString() + "/" + _year.ToString(), CurrencyPair.UNSUPPORTED);
                        ProcessDirectory(_nd);
                    }

                }
                catch (Exception ex)
                {
                    InvokeErrorEvent(ex.Message, CurrencyPair.UNSUPPORTED);

                    if (ex.InnerException != null)
                        InvokeErrorEvent(ex.InnerException.Message, CurrencyPair.UNSUPPORTED);
                }
            }
        }

        private bool LoadAllowed(string filename)
        {


            // USD_GBP_Week4.csv

            string[] _s = filename.Split('_');

            string _t = Shared.NormalizeCurrencyPairName(_s[0] + _s[1]);

            CurrencyPair _p = Shared.GetCurrencyPairID(_t);

            switch (_p)
            {

                case (CurrencyPair.EURUSD):
                    return _loadEURUSD;
                case (CurrencyPair.USDJPY):
                    return _loadUSDJPY;
                case (CurrencyPair.GBPUSD):
                    return _loadGBPUSD;
                case (CurrencyPair.USDCHF):
                    return _loadUSDCHF;
                default:
                    return false;
            }


        }

        private bool LoadAllowed(CurrencyPair p)
        {
            switch (p)
            {

                case (CurrencyPair.EURUSD):
                    return _loadEURUSD;
                case (CurrencyPair.USDJPY):
                    return _loadUSDJPY;
                case (CurrencyPair.GBPUSD):
                    return _loadGBPUSD;
                case (CurrencyPair.USDCHF):
                    return _loadUSDCHF;
                default:
                    return false;
            }
        }

        #endregion


        #region LoadComplete

        protected void _cleanup_DoWork(object sender, System.ComponentModel.DoWorkEventArgs e)
        {


            try
            {
                //_workers.Wait();


                // Something is nuking the Tick Arrays.  I bet its the multithreading.

                for (int x = 0; x < _history.Length; ++x)
                {
                    if (null == _history[x])
                        continue;

                    lock (_completeLock)
                    {
                        if (_history[x].LoadingComplete)
                            continue;

                        _history[x].LoadingComplete = true;
                    }

                    InvokeStatusEvent("Building Tick Data for " + _history[x].Pair.ToString() + " ...", _history[x].Pair);

                    _history[x].LoadComplete();

                    InvokeStatusEvent("Sorting Tick Data for " + _history[x].Pair.ToString() + "...", _history[x].Pair);

                    _history[x].Sort();

                    InvokeStatusEvent("Sorted " + _history[x].Pair.ToString(), _history[x].Pair);

                    InvokeStatusEvent("Verifying Tick Order for " + _history[x].Pair.ToString() + "...", _history[x].Pair);

                    _history[x].Verify();

                    InvokeStatusEvent("Expected Ticks: " + _history[x].ExpectedTicks + "  Actual Ticks: " + _history[x].ActualTicks, _history[x].Pair);

                    InvokeStatusEvent("Duplicates Removed: " + _history[x].DuplicateCount, _history[x].Pair);

                    _history[x].FindGaps();

                    TimeSpan _loadTime = DateTime.Now - _started;

                    InvokeStatusEvent("Loaded in " + _loadTime.Hours.ToString() + ":" + _loadTime.Minutes.ToString() + ":" + _loadTime.Seconds.ToString(), _history[x].Pair);
                    InvokeStatusEvent("Done", _history[x].Pair);

                    GC.Collect();
                }

            }
            catch (Exception ex)
            {
                InvokeErrorEvent(ex.Message, CurrencyPair.UNSUPPORTED);
            }
            finally
            {
                _workers.Release();
            }

        }

        #endregion





        #region Events


        protected void _OnUpdateStatus(object sender, ProgressEventArgs e)
        {
            InvokeStatusEvent(e.Status, e.Pair);
        }

        protected void _OnError(object sender, SimulationErrorEventArgs e)
        {
            InvokeErrorEvent(e.Error, e.Pair);
        }

        protected void InvokeStatusEvent(string status, CurrencyPair pair)
        {
            if (OnUpdateStatus == null) return;

            ProgressEventArgs args = new ProgressEventArgs(status, pair);

            OnUpdateStatus.Raise(null, args);

        }
        public delegate void StatusUpdateHandler(object sender, ProgressEventArgs e);
        public event StatusUpdateHandler OnUpdateStatus;


        protected void InvokeErrorEvent(string error, CurrencyPair pair)
        {
            if (OnError == null) return;

            SimulationErrorEventArgs args = new SimulationErrorEventArgs(error, pair);

            OnError.Raise(null, args);
        }
        public delegate void ErrorHandler(object sender, SimulationErrorEventArgs e);
        public event ErrorHandler OnError;

        protected void InvokeLoadCompleteEvent(object sender)
        {
            if (OnLoadCompleted == null) return;

            EventArgs args = new EventArgs();

            OnLoadCompleted.Raise(sender, args);
        }

        public delegate void OnLoadCompleteHandler(object sender, EventArgs e);
        public event OnLoadCompleteHandler OnLoadCompleted;
        #endregion
    }




}
